#!/usr/bin/env python
# coding=utf-8
#--------------------------------------------------------------------------
# 文件名:       CollectBase.py 
# 说明：        访问基本类 
# 重点说明：    字典 urlDict，穿通整个访问流程
#				urlDict["cookie"]
#				urlDict["method"]
#				urlDict["reurl"]
#				urlDict["url"]
#				urlDict["proxy"]
#
#				outDict["addUrl"]
#				outDict[""]
#				outDict[""]
#
# 版本：        0.01
# 作者：        xiao
#
# 创建时间：    2011-04-27 
# 许可证：      Apache License V2.0
#--------------------------------------------------------------------------

import cookielib
import httplib2
import re
import time
import threading
import random
import os
import urllib2
import urllib
import socket
import XiaoTool as xt

from Queue import Queue

## 线程类
#
class ToThreading(threading.Thread):

	## 线程初始化
	#
	#  @param Name
	#  @param queue
	#  @param Flock
	#  @param parseData
	#  @param cookie
	#
	def __init__(self, Name, queue, Flock, parseData, cookie=None):
		threading.Thread.__init__(self, name=Name)
		self.queue = queue
		self.Flock = Flock

		self.parseData = parseData
		self.cookie = cookie
		self.base = CollectBase()

	#------------------------------------------ End def __init__()

	## 线程运行
	#
	def run(self):
		while True:
			if self.queue.empty():
				break

			urlDict = self.queue.get()
			if "cookie" in urlDict:
				urlDict["cookie"] = self.cookie

			resp, htmlTxt = self.base.getHtml(urlDict)
			if "Set-Cookie" in resp:
				self.cookie = resp["Set-Cookie"]
			if htmlTxt == "":
				continue

			outDict = self.parseData(htmlTxt, resp, urlDict)

			if outDict is not None and "url" in outDict:
				self.queue.put(outDict)

	#------------------------------------------ End def run()
#------------------------------------------------------ End class ToThreading()


## 基础类
#
class CollectBase:

	## 方法注释
	#
	#  @param urlDictList
	#
	def __init__(self, urlDictList={}, timeout=30):
		if len(urlDictList) != 0:
			socket.setdefaulttimeout(timeout)

			self.queue = Queue()
			self.urlDictList = urlDictList
			self.htmlAtion()

	#------------------------------------------ End def __init__()

	## 方法注释
	#
	def htmlAtion(self):
		for urlDict in self.urlDictList:
			self.queue.put(urlDict)
		
		# 启动多线程
		Flock = threading.RLock()
		falge = []
		for i in range(10):
			qi = ToThreading("Collect_Thread-%s" % (i+1), self.queue, Flock, self.parseData)
			falge.append(qi)
			qi.start()
		for qii in falge:
			qii.join()

	#------------------------------------------ End def htmlAtion()

	## 处理返回的数据的方法，继承这个类，重写这个方法
	#
	#  @param htmlTxt
	#  @param resp
	#  @param urlDict
	#  @return
	#
	def parseData(self, htmlTxt="", resp="", urlDict={}):
		return {}
	#------------------------------------------ End def parseData()

	## 主方法，访问网页
	#
	#  @param urlDict
	#  @return
	#
	def getHtml(self, urlDict):
		resp = {"status":"2"}
		if "method" not in urlDict:
			urlDict["method"] = "GET"

		if "proxy" in urlDict:
			try:
				proxy_support = urllib2.ProxyHandler({'http': "http://%s" % urlDict["proxy"]})
				opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
				if "data" in urlDict:
					req = urllib2.Request(urlDict["url"], urlDict["data"], headers = self.setHeaders(urlDict))
				else:
					req = urllib2.Request(urlDict["url"], headers = self.setHeaders(urlDict))

				htmlData = opener.open(req)
				content = htmlData.read()
				resp = {"status":"200"}

			except urllib2.HTTPError, e: # 404等
				resp = {"status":e.code}
				content = ""

			except urllib2.URLError, e:
				xt.setErrLog("URLError-%s: %s \n" % (e.reason, urlDict["url"]))
				content = ""

			except ValueError:
				xt.setErrLog("ValueError: %s \n" % urlDict["url"])
				content = ""

			except:
				xt.setErrLog("OtherError: %s \n" % urlDict["url"])
				content = ""

		else:
			try:
				h = httplib2.Http(".cache")
				
				if "data" in urlDict:
					resp, content = h.request(urlDict["url"], urlDict["method"], body=urlDict["data"], headers = self.setHeaders(urlDict))
				else:
					resp, content = h.request(urlDict["url"], urlDict["method"], headers = self.setHeaders(urlDict))
			except ValueError:
				xt.setErrLog("ValueError: %s \n" % urlDict["url"])
				content = ""
			except:
				xt.setErrLog("OtherError: %s \n" % urlDict["url"])
				content = ""

		if str(resp["status"])[0] not in "23": # 错误，比如：404 错误
			xt.setErrLog("HTTPError-%s: %s \n" % (resp["status"], urlDict["url"]))
			content = ""

		return resp, content

	#------------------------------------------ End def getHtml()

	## HTTP 请求头
	#
	#  @param urlDict
	#  @return
	#
	def setHeaders(self, urlDict):
		accept = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
		acceptL = "en-us"
		acceptE = "gzip, deflate"

		UserAgent = [
				"Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)",
				"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; GTB5; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Maxthon; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)",
				"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB6; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
				"Mozilla/4.0 (compatible; MSIE 8.0; AOL 9.6; AOLBuild 4340.5001; Windows NT 5.1; Trident/4.0)",
				"Mozilla/4.0 (compatible; MSIE 8.0; AOL 9.6; AOLBuild 4340.5000; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
				"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.89; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506)",
				"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; Avant Browser; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
				"Mozilla/5.0 (Windows; U; WinNT; en; rv:1.0.2) Gecko/20030311 Beonex/0.8.2-stable",
				"Mozilla/5.0 (X11; U; Linux i686; nl; rv:1.8.1b2) Gecko/20060821 BonEcho/2.0b2 (Debian-1.99+2.0b2+dfsg-1)",
				"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1b2) Gecko/20060826 BonEcho/2.0b2",
				"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Browzar)",
				"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en; rv:1.9.2.14pre) Gecko/20101212 Camino/2.1a1pre (like Firefox/3.6.14pre)",
				"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en; rv:1.9.0.19) Gecko/2010111021 Camino/2.0.6 (MultiLang) (like Firefox/3.0.19)",
				"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en; rv:1.9.0.18) Gecko/2010021619 Camino/2.0.2 (like Firefox/3.0.18)",
				"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en) AppleWebKit/418.9 (KHTML, like Gecko, Safari) Cheshire/1.0.UNOFFICIAL",
				"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; pl-PL; rv:1.0.1) Gecko/20021111 Chimera/0.6",
				"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.36 (KHTML, like Gecko) Chrome/13.0.766.0 Safari/534.36",
				"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2",
				"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2009011615 Firefox/3.0.5 CometBird/3.0.5",
				"Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Comodo_Dragon/4.1.1.11 Chrome/4.1.249.1042 Safari/532.5",
				"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; pl-pl) AppleWebKit/312.8 (KHTML, like Gecko, Safari) DeskBrowse/1.0",
				"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533+ (KHTML, like Gecko) Element Browser 5.0",
				"Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Epiphany/2.30.6 Safari/534.7",
				"Mozilla/5.0 (X11; Linux x86_64; rv:2.2a1pre) Gecko/20110324 Firefox/4.2a1pre",
				"Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20110506 Firefox/4.0.1",
				"Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20110319 Firefox/4.0",
				"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Flock/3.5.3.4628 Chrome/7.0.517.450 Safari/534.7",
				"Mozilla/5.0 (compatible; IBrowse 3.0; AmigaOS4.0)",
				"iCab/4.7 (Macintosh; U; PPC Mac OS X)",
				"iCab/4.7 (Macintosh; U; Intel Mac OS X)",
				"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
				"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
				"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
				"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
				"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
				"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
				"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)",
				"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
				"Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)",
				"Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; en-US)",
				"Mozilla/4.0 (compatible; MSIE 6.1; Windows XP; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
				"Mozilla/4.0 (compatible; MSIE 6.1; Windows XP)",
				"Mozilla/5.0 (Windows; U; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
				"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/20080722 Firefox/3.0.1 Kapiko/3.0",
				"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)",
				"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en; rv:1.7.12) Gecko/20050928 Firefox/1.0.7 Madfox/3.0",
				"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1",
				"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.4 (KHTML, like Gecko) Maxthon/3.0.6.27 Safari/532.4",
				"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E; Maxthon 2.0)",
				"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; MAXTHON 2.0)"
				]

		uAgent = random.choice(UserAgent)
		headders = {
				"User-Agent":uAgent,
				"Accept":accept,
				"Accept-Language":acceptL,
				"Accept-Encoding":acceptE,
				"Connection":"Keep-Alive"
				}

		if "cookie" in urlDict:
			headders['Cookie'] = urlDict["cookie"]
		if "accept" in urlDict:
			headders["Accept"] = urlDict["accept"]
		if "reurl" in urlDict:
			headders["Referer"] = urlDict["reurl"]
		if "acceptL" in urlDict:
			headders["Accept-Language"] = urlDict["acceptL"]
		if "contentType" in urlDict:
			headders["content-type"] = urlDict["contentType"]

		return headders

	#------------------------------------------ End def setHeaders()
#------------------------------------------------------ End class CollectBase()
